{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-17T08:05:52.994439Z",
     "start_time": "2020-03-17T08:05:52.414147Z"
    }
   },
   "outputs": [],
   "source": [
    "import os\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.metrics import cohen_kappa_score, accuracy_score, mean_absolute_error, f1_score\n",
    "from sklearn.model_selection import GroupKFold, KFold, StratifiedKFold\n",
    "from tqdm import tqdm\n",
    "import lightgbm as lgb\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import math\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import warnings\n",
    "import gc\n",
    "from datetime import datetime\n",
    "\n",
    "warnings.filterwarnings('ignore')\n",
    "pd.set_option('display.max_columns', None)\n",
    "pd.set_option('display.max_rows', None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-17T08:05:52.998069Z",
     "start_time": "2020-03-17T08:05:52.996011Z"
    }
   },
   "outputs": [],
   "source": [
    "seed = 2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-17T08:05:53.684407Z",
     "start_time": "2020-03-17T08:05:52.999310Z"
    }
   },
   "outputs": [],
   "source": [
    "df_train = pd.read_csv('./raw_data/used_car_train_20200313.csv', sep=' ')\n",
    "df_test = pd.read_csv('./raw_data/used_car_testA_20200313.csv', sep=' ')\n",
    "df_sub = pd.read_csv('./raw_data/used_car_sample_submit.csv', sep=' ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-17T08:05:53.724670Z",
     "start_time": "2020-03-17T08:05:53.685690Z"
    }
   },
   "outputs": [],
   "source": [
    "df_feature = pd.concat([df_train, df_test], sort=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-17T08:05:53.749706Z",
     "start_time": "2020-03-17T08:05:53.725932Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SaleID</th>\n",
       "      <th>name</th>\n",
       "      <th>regDate</th>\n",
       "      <th>model</th>\n",
       "      <th>brand</th>\n",
       "      <th>bodyType</th>\n",
       "      <th>fuelType</th>\n",
       "      <th>gearbox</th>\n",
       "      <th>power</th>\n",
       "      <th>kilometer</th>\n",
       "      <th>notRepairedDamage</th>\n",
       "      <th>regionCode</th>\n",
       "      <th>seller</th>\n",
       "      <th>offerType</th>\n",
       "      <th>creatDate</th>\n",
       "      <th>price</th>\n",
       "      <th>v_0</th>\n",
       "      <th>v_1</th>\n",
       "      <th>v_2</th>\n",
       "      <th>v_3</th>\n",
       "      <th>v_4</th>\n",
       "      <th>v_5</th>\n",
       "      <th>v_6</th>\n",
       "      <th>v_7</th>\n",
       "      <th>v_8</th>\n",
       "      <th>v_9</th>\n",
       "      <th>v_10</th>\n",
       "      <th>v_11</th>\n",
       "      <th>v_12</th>\n",
       "      <th>v_13</th>\n",
       "      <th>v_14</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>736</td>\n",
       "      <td>20040402</td>\n",
       "      <td>30.0</td>\n",
       "      <td>6</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>60</td>\n",
       "      <td>12.5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1046</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>20160404</td>\n",
       "      <td>1850.0</td>\n",
       "      <td>43.357796</td>\n",
       "      <td>3.966344</td>\n",
       "      <td>0.050257</td>\n",
       "      <td>2.159744</td>\n",
       "      <td>1.143786</td>\n",
       "      <td>0.235676</td>\n",
       "      <td>0.101988</td>\n",
       "      <td>0.129549</td>\n",
       "      <td>0.022816</td>\n",
       "      <td>0.097462</td>\n",
       "      <td>-2.881803</td>\n",
       "      <td>2.804097</td>\n",
       "      <td>-2.420821</td>\n",
       "      <td>0.795292</td>\n",
       "      <td>0.914762</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>2262</td>\n",
       "      <td>20030301</td>\n",
       "      <td>40.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>-</td>\n",
       "      <td>4366</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>20160309</td>\n",
       "      <td>3600.0</td>\n",
       "      <td>45.305273</td>\n",
       "      <td>5.236112</td>\n",
       "      <td>0.137925</td>\n",
       "      <td>1.380657</td>\n",
       "      <td>-1.422165</td>\n",
       "      <td>0.264777</td>\n",
       "      <td>0.121004</td>\n",
       "      <td>0.135731</td>\n",
       "      <td>0.026597</td>\n",
       "      <td>0.020582</td>\n",
       "      <td>-4.900482</td>\n",
       "      <td>2.096338</td>\n",
       "      <td>-1.030483</td>\n",
       "      <td>-1.722674</td>\n",
       "      <td>0.245522</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>14874</td>\n",
       "      <td>20040403</td>\n",
       "      <td>115.0</td>\n",
       "      <td>15</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>163</td>\n",
       "      <td>12.5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2806</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>20160402</td>\n",
       "      <td>6222.0</td>\n",
       "      <td>45.978359</td>\n",
       "      <td>4.823792</td>\n",
       "      <td>1.319524</td>\n",
       "      <td>-0.998467</td>\n",
       "      <td>-0.996911</td>\n",
       "      <td>0.251410</td>\n",
       "      <td>0.114912</td>\n",
       "      <td>0.165147</td>\n",
       "      <td>0.062173</td>\n",
       "      <td>0.027075</td>\n",
       "      <td>-4.846749</td>\n",
       "      <td>1.803559</td>\n",
       "      <td>1.565330</td>\n",
       "      <td>-0.832687</td>\n",
       "      <td>-0.229963</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>71865</td>\n",
       "      <td>19960908</td>\n",
       "      <td>109.0</td>\n",
       "      <td>10</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>193</td>\n",
       "      <td>15.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>434</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>20160312</td>\n",
       "      <td>2400.0</td>\n",
       "      <td>45.687478</td>\n",
       "      <td>4.492574</td>\n",
       "      <td>-0.050616</td>\n",
       "      <td>0.883600</td>\n",
       "      <td>-2.228079</td>\n",
       "      <td>0.274293</td>\n",
       "      <td>0.110300</td>\n",
       "      <td>0.121964</td>\n",
       "      <td>0.033395</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-4.509599</td>\n",
       "      <td>1.285940</td>\n",
       "      <td>-0.501868</td>\n",
       "      <td>-2.438353</td>\n",
       "      <td>-0.478699</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>111080</td>\n",
       "      <td>20120103</td>\n",
       "      <td>110.0</td>\n",
       "      <td>5</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>68</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6977</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>20160313</td>\n",
       "      <td>5200.0</td>\n",
       "      <td>44.383511</td>\n",
       "      <td>2.031433</td>\n",
       "      <td>0.572169</td>\n",
       "      <td>-1.571239</td>\n",
       "      <td>2.246088</td>\n",
       "      <td>0.228036</td>\n",
       "      <td>0.073205</td>\n",
       "      <td>0.091880</td>\n",
       "      <td>0.078819</td>\n",
       "      <td>0.121534</td>\n",
       "      <td>-1.896240</td>\n",
       "      <td>0.910783</td>\n",
       "      <td>0.931110</td>\n",
       "      <td>2.834518</td>\n",
       "      <td>1.923482</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   SaleID    name   regDate  model  brand  bodyType  fuelType  gearbox  power  \\\n",
       "0       0     736  20040402   30.0      6       1.0       0.0      0.0     60   \n",
       "1       1    2262  20030301   40.0      1       2.0       0.0      0.0      0   \n",
       "2       2   14874  20040403  115.0     15       1.0       0.0      0.0    163   \n",
       "3       3   71865  19960908  109.0     10       0.0       0.0      1.0    193   \n",
       "4       4  111080  20120103  110.0      5       1.0       0.0      0.0     68   \n",
       "\n",
       "   kilometer notRepairedDamage  regionCode  seller  offerType  creatDate  \\\n",
       "0       12.5               0.0        1046       0          0   20160404   \n",
       "1       15.0                 -        4366       0          0   20160309   \n",
       "2       12.5               0.0        2806       0          0   20160402   \n",
       "3       15.0               0.0         434       0          0   20160312   \n",
       "4        5.0               0.0        6977       0          0   20160313   \n",
       "\n",
       "    price        v_0       v_1       v_2       v_3       v_4       v_5  \\\n",
       "0  1850.0  43.357796  3.966344  0.050257  2.159744  1.143786  0.235676   \n",
       "1  3600.0  45.305273  5.236112  0.137925  1.380657 -1.422165  0.264777   \n",
       "2  6222.0  45.978359  4.823792  1.319524 -0.998467 -0.996911  0.251410   \n",
       "3  2400.0  45.687478  4.492574 -0.050616  0.883600 -2.228079  0.274293   \n",
       "4  5200.0  44.383511  2.031433  0.572169 -1.571239  2.246088  0.228036   \n",
       "\n",
       "        v_6       v_7       v_8       v_9      v_10      v_11      v_12  \\\n",
       "0  0.101988  0.129549  0.022816  0.097462 -2.881803  2.804097 -2.420821   \n",
       "1  0.121004  0.135731  0.026597  0.020582 -4.900482  2.096338 -1.030483   \n",
       "2  0.114912  0.165147  0.062173  0.027075 -4.846749  1.803559  1.565330   \n",
       "3  0.110300  0.121964  0.033395  0.000000 -4.509599  1.285940 -0.501868   \n",
       "4  0.073205  0.091880  0.078819  0.121534 -1.896240  0.910783  0.931110   \n",
       "\n",
       "       v_13      v_14  \n",
       "0  0.795292  0.914762  \n",
       "1 -1.722674  0.245522  \n",
       "2 -0.832687 -0.229963  \n",
       "3 -2.438353 -0.478699  \n",
       "4  2.834518  1.923482  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_feature.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-17T08:05:53.768431Z",
     "start_time": "2020-03-17T08:05:53.750700Z"
    }
   },
   "outputs": [],
   "source": [
    "df_feature['notRepairedDamage'] = df_feature['notRepairedDamage'].replace(\n",
    "    '-', 2)\n",
    "df_feature['notRepairedDamage'] = df_feature['notRepairedDamage'].astype(\n",
    "    'float')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-17T08:05:53.836162Z",
     "start_time": "2020-03-17T08:05:53.770578Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SaleID</th>\n",
       "      <th>name</th>\n",
       "      <th>regDate</th>\n",
       "      <th>model</th>\n",
       "      <th>brand</th>\n",
       "      <th>bodyType</th>\n",
       "      <th>fuelType</th>\n",
       "      <th>gearbox</th>\n",
       "      <th>power</th>\n",
       "      <th>kilometer</th>\n",
       "      <th>notRepairedDamage</th>\n",
       "      <th>regionCode</th>\n",
       "      <th>seller</th>\n",
       "      <th>offerType</th>\n",
       "      <th>creatDate</th>\n",
       "      <th>price</th>\n",
       "      <th>v_0</th>\n",
       "      <th>v_1</th>\n",
       "      <th>v_2</th>\n",
       "      <th>v_3</th>\n",
       "      <th>v_4</th>\n",
       "      <th>v_5</th>\n",
       "      <th>v_6</th>\n",
       "      <th>v_7</th>\n",
       "      <th>v_8</th>\n",
       "      <th>v_9</th>\n",
       "      <th>v_10</th>\n",
       "      <th>v_11</th>\n",
       "      <th>v_12</th>\n",
       "      <th>v_13</th>\n",
       "      <th>v_14</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>49995</th>\n",
       "      <td>199995</td>\n",
       "      <td>20903</td>\n",
       "      <td>19960503</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>116</td>\n",
       "      <td>15.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3219</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>20160320</td>\n",
       "      <td>NaN</td>\n",
       "      <td>45.621391</td>\n",
       "      <td>5.958453</td>\n",
       "      <td>-0.918571</td>\n",
       "      <td>0.774826</td>\n",
       "      <td>-2.021739</td>\n",
       "      <td>0.284664</td>\n",
       "      <td>0.130044</td>\n",
       "      <td>0.049833</td>\n",
       "      <td>0.028807</td>\n",
       "      <td>0.004616</td>\n",
       "      <td>-5.978511</td>\n",
       "      <td>1.303174</td>\n",
       "      <td>-1.207191</td>\n",
       "      <td>-1.981240</td>\n",
       "      <td>-0.357695</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49996</th>\n",
       "      <td>199996</td>\n",
       "      <td>708</td>\n",
       "      <td>19991011</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>75</td>\n",
       "      <td>15.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1857</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>20160329</td>\n",
       "      <td>NaN</td>\n",
       "      <td>43.935162</td>\n",
       "      <td>4.476841</td>\n",
       "      <td>-0.841710</td>\n",
       "      <td>1.328253</td>\n",
       "      <td>-1.292675</td>\n",
       "      <td>0.268101</td>\n",
       "      <td>0.108095</td>\n",
       "      <td>0.066039</td>\n",
       "      <td>0.025468</td>\n",
       "      <td>0.025971</td>\n",
       "      <td>-3.913825</td>\n",
       "      <td>1.759524</td>\n",
       "      <td>-2.075658</td>\n",
       "      <td>-1.154847</td>\n",
       "      <td>0.169073</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49997</th>\n",
       "      <td>199997</td>\n",
       "      <td>6693</td>\n",
       "      <td>20040412</td>\n",
       "      <td>49.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>224</td>\n",
       "      <td>15.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3452</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>20160305</td>\n",
       "      <td>NaN</td>\n",
       "      <td>46.537137</td>\n",
       "      <td>4.170806</td>\n",
       "      <td>0.388595</td>\n",
       "      <td>-0.704689</td>\n",
       "      <td>-1.480710</td>\n",
       "      <td>0.269432</td>\n",
       "      <td>0.105724</td>\n",
       "      <td>0.117652</td>\n",
       "      <td>0.057479</td>\n",
       "      <td>0.015669</td>\n",
       "      <td>-4.639065</td>\n",
       "      <td>0.654713</td>\n",
       "      <td>1.137756</td>\n",
       "      <td>-1.390531</td>\n",
       "      <td>0.254420</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49998</th>\n",
       "      <td>199998</td>\n",
       "      <td>96900</td>\n",
       "      <td>20020008</td>\n",
       "      <td>27.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>334</td>\n",
       "      <td>15.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1998</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>20160404</td>\n",
       "      <td>NaN</td>\n",
       "      <td>46.771359</td>\n",
       "      <td>-3.296814</td>\n",
       "      <td>0.243566</td>\n",
       "      <td>-1.277411</td>\n",
       "      <td>-0.404881</td>\n",
       "      <td>0.261152</td>\n",
       "      <td>0.000490</td>\n",
       "      <td>0.137366</td>\n",
       "      <td>0.086216</td>\n",
       "      <td>0.051383</td>\n",
       "      <td>1.833504</td>\n",
       "      <td>-2.828687</td>\n",
       "      <td>2.465630</td>\n",
       "      <td>-0.911682</td>\n",
       "      <td>-2.057353</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49999</th>\n",
       "      <td>199999</td>\n",
       "      <td>193384</td>\n",
       "      <td>20041109</td>\n",
       "      <td>166.0</td>\n",
       "      <td>6</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>68</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3276</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>20160322</td>\n",
       "      <td>NaN</td>\n",
       "      <td>43.731010</td>\n",
       "      <td>-3.121867</td>\n",
       "      <td>0.027348</td>\n",
       "      <td>-0.808914</td>\n",
       "      <td>2.116551</td>\n",
       "      <td>0.228730</td>\n",
       "      <td>0.000300</td>\n",
       "      <td>0.103534</td>\n",
       "      <td>0.080625</td>\n",
       "      <td>0.124264</td>\n",
       "      <td>2.914571</td>\n",
       "      <td>-1.135270</td>\n",
       "      <td>0.547628</td>\n",
       "      <td>2.094057</td>\n",
       "      <td>-1.552150</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       SaleID    name   regDate  model  brand  bodyType  fuelType  gearbox  \\\n",
       "49995  199995   20903  19960503    4.0      4       4.0       0.0      0.0   \n",
       "49996  199996     708  19991011    0.0      0       0.0       0.0      0.0   \n",
       "49997  199997    6693  20040412   49.0      1       0.0       1.0      1.0   \n",
       "49998  199998   96900  20020008   27.0      1       0.0       0.0      1.0   \n",
       "49999  199999  193384  20041109  166.0      6       1.0       NaN      1.0   \n",
       "\n",
       "       power  kilometer  notRepairedDamage  regionCode  seller  offerType  \\\n",
       "49995    116       15.0                0.0        3219       0          0   \n",
       "49996     75       15.0                0.0        1857       0          0   \n",
       "49997    224       15.0                0.0        3452       0          0   \n",
       "49998    334       15.0                0.0        1998       0          0   \n",
       "49999     68        9.0                0.0        3276       0          0   \n",
       "\n",
       "       creatDate  price        v_0       v_1       v_2       v_3       v_4  \\\n",
       "49995   20160320    NaN  45.621391  5.958453 -0.918571  0.774826 -2.021739   \n",
       "49996   20160329    NaN  43.935162  4.476841 -0.841710  1.328253 -1.292675   \n",
       "49997   20160305    NaN  46.537137  4.170806  0.388595 -0.704689 -1.480710   \n",
       "49998   20160404    NaN  46.771359 -3.296814  0.243566 -1.277411 -0.404881   \n",
       "49999   20160322    NaN  43.731010 -3.121867  0.027348 -0.808914  2.116551   \n",
       "\n",
       "            v_5       v_6       v_7       v_8       v_9      v_10      v_11  \\\n",
       "49995  0.284664  0.130044  0.049833  0.028807  0.004616 -5.978511  1.303174   \n",
       "49996  0.268101  0.108095  0.066039  0.025468  0.025971 -3.913825  1.759524   \n",
       "49997  0.269432  0.105724  0.117652  0.057479  0.015669 -4.639065  0.654713   \n",
       "49998  0.261152  0.000490  0.137366  0.086216  0.051383  1.833504 -2.828687   \n",
       "49999  0.228730  0.000300  0.103534  0.080625  0.124264  2.914571 -1.135270   \n",
       "\n",
       "           v_12      v_13      v_14  \n",
       "49995 -1.207191 -1.981240 -0.357695  \n",
       "49996 -2.075658 -1.154847  0.169073  \n",
       "49997  1.137756 -1.390531  0.254420  \n",
       "49998  2.465630 -0.911682 -2.057353  \n",
       "49999  0.547628  2.094057 -1.552150  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_feature.tail()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# feature engine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-17T08:05:53.843194Z",
     "start_time": "2020-03-17T08:05:53.837355Z"
    }
   },
   "outputs": [],
   "source": [
    "del df_feature['seller']\n",
    "del df_feature['offerType']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-17T08:05:53.909955Z",
     "start_time": "2020-03-17T08:05:53.844249Z"
    }
   },
   "outputs": [],
   "source": [
    "df_feature['price'] = np.log1p(df_feature['price'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-17T08:05:53.988453Z",
     "start_time": "2020-03-17T08:05:53.912470Z"
    }
   },
   "outputs": [],
   "source": [
    "df_feature['name_count'] = df_feature.groupby(\n",
    "    ['name'])['SaleID'].transform('count')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-17T08:05:54.608585Z",
     "start_time": "2020-03-17T08:05:53.989672Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def date_parse(x):\n",
    "    year = int(str(x)[:4])\n",
    "    month = int(str(x)[4:6])\n",
    "    day = int(str(x)[6:8])\n",
    "\n",
    "    if month < 1:\n",
    "        month = 1\n",
    "\n",
    "    date = datetime(year, month, day)\n",
    "    return date\n",
    "\n",
    "\n",
    "df_feature['regDate'] = df_feature['regDate'].apply(date_parse)\n",
    "df_feature['creatDate'] = df_feature['creatDate'].apply(date_parse)\n",
    "df_feature['regDate_year'] = df_feature['regDate'].dt.year"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-17T08:05:54.621341Z",
     "start_time": "2020-03-17T08:05:54.609999Z"
    }
   },
   "outputs": [],
   "source": [
    "df_feature['car_age_day'] = (\n",
    "    df_feature['creatDate'] - df_feature['regDate']).dt.days\n",
    "df_feature['car_age_year'] = round(df_feature['car_age_day'] / 365, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-17T08:05:54.625958Z",
     "start_time": "2020-03-17T08:05:54.622490Z"
    }
   },
   "outputs": [],
   "source": [
    "# 简单统计\n",
    "def stat(df, df_merge, group_by, agg):\n",
    "    group = df.groupby(group_by).agg(agg)\n",
    "\n",
    "    columns = []\n",
    "    for on, methods in agg.items():\n",
    "        for method in methods:\n",
    "            columns.append('{}_{}_{}'.format('_'.join(group_by), on, method))\n",
    "    group.columns = columns\n",
    "    group.reset_index(inplace=True)\n",
    "    df_merge = df_merge.merge(group, on=group_by, how='left')\n",
    "\n",
    "    del (group)\n",
    "    gc.collect()\n",
    "    return df_merge"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-17T08:05:54.630375Z",
     "start_time": "2020-03-17T08:05:54.627097Z"
    }
   },
   "outputs": [],
   "source": [
    "def statis_feat(df_know, df_unknow):\n",
    "    df_unknow = stat(df_know, df_unknow, ['model'], {'price': ['mean']})\n",
    "    df_unknow = stat(df_know, df_unknow, ['regionCode'], {'price': ['mean']})\n",
    "    df_unknow = stat(df_know, df_unknow, ['name'], {'price': ['mean']})\n",
    "\n",
    "    return df_unknow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-17T08:05:56.303780Z",
     "start_time": "2020-03-17T08:05:54.631510Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 5折交叉\n",
    "df_train = df_feature[~df_feature['price'].isnull()]\n",
    "df_train = df_train.reset_index(drop=True)\n",
    "df_test = df_feature[df_feature['price'].isnull()]\n",
    "\n",
    "df_stas_feat = None\n",
    "kf = KFold(n_splits=5, random_state=seed, shuffle=True)\n",
    "for train_index, val_index in kf.split(df_train):\n",
    "    df_fold_train = df_train.iloc[train_index]\n",
    "    df_fold_val = df_train.iloc[val_index]\n",
    "\n",
    "    df_fold_val = statis_feat(df_fold_train, df_fold_val)\n",
    "    df_stas_feat = pd.concat([df_stas_feat, df_fold_val], axis=0)\n",
    "\n",
    "    del(df_fold_train)\n",
    "    del(df_fold_val)\n",
    "    gc.collect()\n",
    "\n",
    "df_test = statis_feat(df_train, df_test)\n",
    "df_feature = pd.concat([df_stas_feat, df_test], axis=0)\n",
    "\n",
    "del(df_stas_feat)\n",
    "del(df_train)\n",
    "del(df_test)\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-17T08:05:56.329459Z",
     "start_time": "2020-03-17T08:05:56.304965Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SaleID</th>\n",
       "      <th>name</th>\n",
       "      <th>regDate</th>\n",
       "      <th>model</th>\n",
       "      <th>brand</th>\n",
       "      <th>bodyType</th>\n",
       "      <th>fuelType</th>\n",
       "      <th>gearbox</th>\n",
       "      <th>power</th>\n",
       "      <th>kilometer</th>\n",
       "      <th>notRepairedDamage</th>\n",
       "      <th>regionCode</th>\n",
       "      <th>creatDate</th>\n",
       "      <th>price</th>\n",
       "      <th>v_0</th>\n",
       "      <th>v_1</th>\n",
       "      <th>v_2</th>\n",
       "      <th>v_3</th>\n",
       "      <th>v_4</th>\n",
       "      <th>v_5</th>\n",
       "      <th>v_6</th>\n",
       "      <th>v_7</th>\n",
       "      <th>v_8</th>\n",
       "      <th>v_9</th>\n",
       "      <th>v_10</th>\n",
       "      <th>v_11</th>\n",
       "      <th>v_12</th>\n",
       "      <th>v_13</th>\n",
       "      <th>v_14</th>\n",
       "      <th>name_count</th>\n",
       "      <th>regDate_year</th>\n",
       "      <th>car_age_day</th>\n",
       "      <th>car_age_year</th>\n",
       "      <th>model_price_mean</th>\n",
       "      <th>regionCode_price_mean</th>\n",
       "      <th>name_price_mean</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3</td>\n",
       "      <td>71865</td>\n",
       "      <td>1996-09-08</td>\n",
       "      <td>109.0</td>\n",
       "      <td>10</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>193</td>\n",
       "      <td>15.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>434</td>\n",
       "      <td>2016-03-12</td>\n",
       "      <td>7.783641</td>\n",
       "      <td>45.687478</td>\n",
       "      <td>4.492574</td>\n",
       "      <td>-0.050616</td>\n",
       "      <td>0.883600</td>\n",
       "      <td>-2.228079</td>\n",
       "      <td>0.274293</td>\n",
       "      <td>0.110300</td>\n",
       "      <td>0.121964</td>\n",
       "      <td>0.033395</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-4.509599</td>\n",
       "      <td>1.285940</td>\n",
       "      <td>-0.501868</td>\n",
       "      <td>-2.438353</td>\n",
       "      <td>-0.478699</td>\n",
       "      <td>2</td>\n",
       "      <td>1996</td>\n",
       "      <td>7125</td>\n",
       "      <td>19.5</td>\n",
       "      <td>9.063339</td>\n",
       "      <td>8.013673</td>\n",
       "      <td>8.682877</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7</td>\n",
       "      <td>165346</td>\n",
       "      <td>1999-07-06</td>\n",
       "      <td>26.0</td>\n",
       "      <td>14</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>101</td>\n",
       "      <td>15.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4000</td>\n",
       "      <td>2016-03-26</td>\n",
       "      <td>6.908755</td>\n",
       "      <td>42.255586</td>\n",
       "      <td>-3.167771</td>\n",
       "      <td>-0.676693</td>\n",
       "      <td>1.942673</td>\n",
       "      <td>0.524206</td>\n",
       "      <td>0.239506</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.122943</td>\n",
       "      <td>0.039839</td>\n",
       "      <td>0.082413</td>\n",
       "      <td>3.693829</td>\n",
       "      <td>-0.245014</td>\n",
       "      <td>-2.192810</td>\n",
       "      <td>0.236728</td>\n",
       "      <td>0.195567</td>\n",
       "      <td>1</td>\n",
       "      <td>1999</td>\n",
       "      <td>6108</td>\n",
       "      <td>16.7</td>\n",
       "      <td>7.566696</td>\n",
       "      <td>8.281427</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>12</td>\n",
       "      <td>120103</td>\n",
       "      <td>2001-03-07</td>\n",
       "      <td>48.0</td>\n",
       "      <td>14</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>58</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2753</td>\n",
       "      <td>2016-03-21</td>\n",
       "      <td>7.378384</td>\n",
       "      <td>42.309224</td>\n",
       "      <td>-3.082286</td>\n",
       "      <td>-0.604813</td>\n",
       "      <td>0.843333</td>\n",
       "      <td>0.388727</td>\n",
       "      <td>0.240775</td>\n",
       "      <td>0.000116</td>\n",
       "      <td>0.104573</td>\n",
       "      <td>0.053303</td>\n",
       "      <td>0.074250</td>\n",
       "      <td>3.477291</td>\n",
       "      <td>-0.461450</td>\n",
       "      <td>-1.442835</td>\n",
       "      <td>0.659255</td>\n",
       "      <td>1.199350</td>\n",
       "      <td>1</td>\n",
       "      <td>2001</td>\n",
       "      <td>5493</td>\n",
       "      <td>15.0</td>\n",
       "      <td>7.092135</td>\n",
       "      <td>8.661780</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>16</td>\n",
       "      <td>10036</td>\n",
       "      <td>2011-09-11</td>\n",
       "      <td>105.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>239</td>\n",
       "      <td>12.5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>419</td>\n",
       "      <td>2016-03-06</td>\n",
       "      <td>9.259226</td>\n",
       "      <td>48.307770</td>\n",
       "      <td>2.366464</td>\n",
       "      <td>1.160401</td>\n",
       "      <td>-1.641052</td>\n",
       "      <td>0.940788</td>\n",
       "      <td>0.251404</td>\n",
       "      <td>0.082237</td>\n",
       "      <td>0.150080</td>\n",
       "      <td>0.082606</td>\n",
       "      <td>0.088695</td>\n",
       "      <td>-3.625918</td>\n",
       "      <td>-0.621775</td>\n",
       "      <td>3.086576</td>\n",
       "      <td>0.165461</td>\n",
       "      <td>-2.192635</td>\n",
       "      <td>18</td>\n",
       "      <td>2011</td>\n",
       "      <td>1638</td>\n",
       "      <td>4.5</td>\n",
       "      <td>9.912501</td>\n",
       "      <td>9.273880</td>\n",
       "      <td>9.242547</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>23</td>\n",
       "      <td>8949</td>\n",
       "      <td>1994-04-01</td>\n",
       "      <td>78.0</td>\n",
       "      <td>7</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>105</td>\n",
       "      <td>15.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1266</td>\n",
       "      <td>2016-03-17</td>\n",
       "      <td>6.396930</td>\n",
       "      <td>43.740185</td>\n",
       "      <td>3.408253</td>\n",
       "      <td>-1.850466</td>\n",
       "      <td>2.593211</td>\n",
       "      <td>0.749961</td>\n",
       "      <td>0.263572</td>\n",
       "      <td>0.093292</td>\n",
       "      <td>0.016425</td>\n",
       "      <td>0.013495</td>\n",
       "      <td>0.094000</td>\n",
       "      <td>-2.891659</td>\n",
       "      <td>1.104114</td>\n",
       "      <td>-3.580304</td>\n",
       "      <td>0.157992</td>\n",
       "      <td>-1.133201</td>\n",
       "      <td>12</td>\n",
       "      <td>1994</td>\n",
       "      <td>8021</td>\n",
       "      <td>22.0</td>\n",
       "      <td>7.744373</td>\n",
       "      <td>8.087276</td>\n",
       "      <td>6.680958</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   SaleID    name    regDate  model  brand  bodyType  fuelType  gearbox  \\\n",
       "0       3   71865 1996-09-08  109.0     10       0.0       0.0      1.0   \n",
       "1       7  165346 1999-07-06   26.0     14       1.0       0.0      0.0   \n",
       "2      12  120103 2001-03-07   48.0     14       1.0       0.0      0.0   \n",
       "3      16   10036 2011-09-11  105.0      1       0.0       1.0      1.0   \n",
       "4      23    8949 1994-04-01   78.0      7       5.0       0.0      0.0   \n",
       "\n",
       "   power  kilometer  notRepairedDamage  regionCode  creatDate     price  \\\n",
       "0    193       15.0                0.0         434 2016-03-12  7.783641   \n",
       "1    101       15.0                0.0        4000 2016-03-26  6.908755   \n",
       "2     58        6.0                0.0        2753 2016-03-21  7.378384   \n",
       "3    239       12.5                0.0         419 2016-03-06  9.259226   \n",
       "4    105       15.0                1.0        1266 2016-03-17  6.396930   \n",
       "\n",
       "         v_0       v_1       v_2       v_3       v_4       v_5       v_6  \\\n",
       "0  45.687478  4.492574 -0.050616  0.883600 -2.228079  0.274293  0.110300   \n",
       "1  42.255586 -3.167771 -0.676693  1.942673  0.524206  0.239506  0.000000   \n",
       "2  42.309224 -3.082286 -0.604813  0.843333  0.388727  0.240775  0.000116   \n",
       "3  48.307770  2.366464  1.160401 -1.641052  0.940788  0.251404  0.082237   \n",
       "4  43.740185  3.408253 -1.850466  2.593211  0.749961  0.263572  0.093292   \n",
       "\n",
       "        v_7       v_8       v_9      v_10      v_11      v_12      v_13  \\\n",
       "0  0.121964  0.033395  0.000000 -4.509599  1.285940 -0.501868 -2.438353   \n",
       "1  0.122943  0.039839  0.082413  3.693829 -0.245014 -2.192810  0.236728   \n",
       "2  0.104573  0.053303  0.074250  3.477291 -0.461450 -1.442835  0.659255   \n",
       "3  0.150080  0.082606  0.088695 -3.625918 -0.621775  3.086576  0.165461   \n",
       "4  0.016425  0.013495  0.094000 -2.891659  1.104114 -3.580304  0.157992   \n",
       "\n",
       "       v_14  name_count  regDate_year  car_age_day  car_age_year  \\\n",
       "0 -0.478699           2          1996         7125          19.5   \n",
       "1  0.195567           1          1999         6108          16.7   \n",
       "2  1.199350           1          2001         5493          15.0   \n",
       "3 -2.192635          18          2011         1638           4.5   \n",
       "4 -1.133201          12          1994         8021          22.0   \n",
       "\n",
       "   model_price_mean  regionCode_price_mean  name_price_mean  \n",
       "0          9.063339               8.013673         8.682877  \n",
       "1          7.566696               8.281427              NaN  \n",
       "2          7.092135               8.661780              NaN  \n",
       "3          9.912501               9.273880         9.242547  \n",
       "4          7.744373               8.087276         6.680958  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_feature.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-17T08:05:56.385356Z",
     "start_time": "2020-03-17T08:05:56.330468Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/200000 [00:00<?, ?it/s]\n"
     ]
    }
   ],
   "source": [
    "from sklearn.preprocessing import LabelEncoder\n",
    "for f in tqdm(df_feature.select_dtypes('object')):\n",
    "    lbl = LabelEncoder()\n",
    "    df_feature[f] = lbl.fit_transform(df_feature[f].astype(str))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-17T08:05:56.426186Z",
     "start_time": "2020-03-17T08:05:56.388061Z"
    }
   },
   "outputs": [],
   "source": [
    "df_test = df_feature[df_feature['price'].isnull()].copy()\n",
    "df_train = df_feature[df_feature['price'].notnull()].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-17T08:08:36.115209Z",
     "start_time": "2020-03-17T08:05:56.427354Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Fold_1 Training ================================\n",
      "\n",
      "Training until validation scores don't improve for 50 rounds\n",
      "[500]\ttrain's l1: 0.0906478\tvalid's l1: 0.120959\n",
      "[1000]\ttrain's l1: 0.0715953\tvalid's l1: 0.117581\n",
      "[1500]\ttrain's l1: 0.0591564\tvalid's l1: 0.116015\n",
      "[2000]\ttrain's l1: 0.0501136\tvalid's l1: 0.114961\n",
      "[2500]\ttrain's l1: 0.0432541\tvalid's l1: 0.114241\n",
      "[3000]\ttrain's l1: 0.0378658\tvalid's l1: 0.113719\n",
      "[3500]\ttrain's l1: 0.033574\tvalid's l1: 0.113276\n",
      "[4000]\ttrain's l1: 0.0301141\tvalid's l1: 0.112923\n",
      "[4500]\ttrain's l1: 0.0272713\tvalid's l1: 0.11264\n",
      "[5000]\ttrain's l1: 0.0248275\tvalid's l1: 0.112463\n",
      "[5500]\ttrain's l1: 0.0227752\tvalid's l1: 0.112264\n",
      "[6000]\ttrain's l1: 0.0210319\tvalid's l1: 0.112128\n",
      "Early stopping, best iteration is:\n",
      "[6082]\ttrain's l1: 0.0207681\tvalid's l1: 0.11211\n",
      "\n",
      "Fold_2 Training ================================\n",
      "\n",
      "Training until validation scores don't improve for 50 rounds\n",
      "[500]\ttrain's l1: 0.0908584\tvalid's l1: 0.120676\n",
      "[1000]\ttrain's l1: 0.0715355\tvalid's l1: 0.116882\n",
      "[1500]\ttrain's l1: 0.0591691\tvalid's l1: 0.11509\n",
      "[2000]\ttrain's l1: 0.0501004\tvalid's l1: 0.114023\n",
      "[2500]\ttrain's l1: 0.0433727\tvalid's l1: 0.113249\n",
      "[3000]\ttrain's l1: 0.0379434\tvalid's l1: 0.11277\n",
      "[3500]\ttrain's l1: 0.0336627\tvalid's l1: 0.112429\n",
      "[4000]\ttrain's l1: 0.0301741\tvalid's l1: 0.112049\n",
      "[4500]\ttrain's l1: 0.0272464\tvalid's l1: 0.111776\n",
      "[5000]\ttrain's l1: 0.0248089\tvalid's l1: 0.111568\n",
      "[5500]\ttrain's l1: 0.0227979\tvalid's l1: 0.111396\n",
      "[6000]\ttrain's l1: 0.0210145\tvalid's l1: 0.111287\n",
      "[6500]\ttrain's l1: 0.0194832\tvalid's l1: 0.111182\n",
      "[7000]\ttrain's l1: 0.0181121\tvalid's l1: 0.111095\n",
      "Early stopping, best iteration is:\n",
      "[7069]\ttrain's l1: 0.0179486\tvalid's l1: 0.111079\n",
      "\n",
      "Fold_3 Training ================================\n",
      "\n",
      "Training until validation scores don't improve for 50 rounds\n",
      "[500]\ttrain's l1: 0.0905515\tvalid's l1: 0.120055\n",
      "[1000]\ttrain's l1: 0.0712694\tvalid's l1: 0.116975\n",
      "[1500]\ttrain's l1: 0.0588194\tvalid's l1: 0.115401\n",
      "[2000]\ttrain's l1: 0.0498159\tvalid's l1: 0.114592\n",
      "[2500]\ttrain's l1: 0.0430531\tvalid's l1: 0.113925\n",
      "[3000]\ttrain's l1: 0.0376828\tvalid's l1: 0.113577\n",
      "[3500]\ttrain's l1: 0.0334282\tvalid's l1: 0.11319\n",
      "[4000]\ttrain's l1: 0.029869\tvalid's l1: 0.112876\n",
      "[4500]\ttrain's l1: 0.0270219\tvalid's l1: 0.112646\n",
      "Early stopping, best iteration is:\n",
      "[4595]\ttrain's l1: 0.0265173\tvalid's l1: 0.112612\n",
      "\n",
      "Fold_4 Training ================================\n",
      "\n",
      "Training until validation scores don't improve for 50 rounds\n",
      "[500]\ttrain's l1: 0.0909389\tvalid's l1: 0.12062\n",
      "[1000]\ttrain's l1: 0.0718913\tvalid's l1: 0.116962\n",
      "[1500]\ttrain's l1: 0.0595433\tvalid's l1: 0.11531\n",
      "[2000]\ttrain's l1: 0.0505178\tvalid's l1: 0.114134\n",
      "[2500]\ttrain's l1: 0.0435231\tvalid's l1: 0.113474\n",
      "[3000]\ttrain's l1: 0.0381091\tvalid's l1: 0.112888\n",
      "[3500]\ttrain's l1: 0.0336938\tvalid's l1: 0.112473\n",
      "[4000]\ttrain's l1: 0.0301496\tvalid's l1: 0.112124\n",
      "[4500]\ttrain's l1: 0.0272855\tvalid's l1: 0.111895\n",
      "[5000]\ttrain's l1: 0.024796\tvalid's l1: 0.111725\n",
      "[5500]\ttrain's l1: 0.0226996\tvalid's l1: 0.111587\n",
      "[6000]\ttrain's l1: 0.0210054\tvalid's l1: 0.111434\n",
      "Early stopping, best iteration is:\n",
      "[6129]\ttrain's l1: 0.0206095\tvalid's l1: 0.11141\n",
      "\n",
      "Fold_5 Training ================================\n",
      "\n",
      "Training until validation scores don't improve for 50 rounds\n",
      "[500]\ttrain's l1: 0.0905834\tvalid's l1: 0.120096\n",
      "[1000]\ttrain's l1: 0.0714609\tvalid's l1: 0.116853\n",
      "[1500]\ttrain's l1: 0.0590695\tvalid's l1: 0.11518\n",
      "[2000]\ttrain's l1: 0.0500012\tvalid's l1: 0.11428\n",
      "[2500]\ttrain's l1: 0.0431355\tvalid's l1: 0.113731\n",
      "[3000]\ttrain's l1: 0.0377669\tvalid's l1: 0.113179\n",
      "[3500]\ttrain's l1: 0.0334865\tvalid's l1: 0.112781\n",
      "[4000]\ttrain's l1: 0.0300256\tvalid's l1: 0.112456\n",
      "[4500]\ttrain's l1: 0.0272288\tvalid's l1: 0.112269\n",
      "[5000]\ttrain's l1: 0.0247598\tvalid's l1: 0.112083\n",
      "[5500]\ttrain's l1: 0.0227094\tvalid's l1: 0.111931\n",
      "[6000]\ttrain's l1: 0.0209011\tvalid's l1: 0.111798\n",
      "Early stopping, best iteration is:\n",
      "[6038]\ttrain's l1: 0.0207821\tvalid's l1: 0.111779\n"
     ]
    }
   ],
   "source": [
    "ycol = 'price'\n",
    "feature_names = list(\n",
    "    filter(lambda x: x not in [ycol, 'SaleID', 'regDate', 'creatDate'], df_train.columns))\n",
    "\n",
    "model = lgb.LGBMRegressor(num_leaves=64,\n",
    "                          max_depth=10,\n",
    "                          learning_rate=0.1,\n",
    "                          n_estimators=10000000,\n",
    "                          subsample=0.8,\n",
    "                          feature_fraction=0.8,\n",
    "                          reg_alpha=0.5,\n",
    "                          reg_lambda=0.5,\n",
    "                          random_state=seed,\n",
    "                          metric=None\n",
    "                          )\n",
    "\n",
    "\n",
    "oof = []\n",
    "prediction = df_test[['SaleID']]\n",
    "prediction['price'] = 0\n",
    "df_importance_list = []\n",
    "\n",
    "kfold = KFold(n_splits=5, shuffle=False, random_state=seed)\n",
    "for fold_id, (trn_idx, val_idx) in enumerate(kfold.split(df_train[feature_names])):\n",
    "    X_train = df_train.iloc[trn_idx][feature_names]\n",
    "    Y_train = df_train.iloc[trn_idx][ycol]\n",
    "\n",
    "    X_val = df_train.iloc[val_idx][feature_names]\n",
    "    Y_val = df_train.iloc[val_idx][ycol]\n",
    "\n",
    "    print('\\nFold_{} Training ================================\\n'.format(fold_id+1))\n",
    "\n",
    "    lgb_model = model.fit(X_train,\n",
    "                          Y_train,\n",
    "                          eval_names=['train', 'valid'],\n",
    "                          eval_set=[(X_train, Y_train), (X_val, Y_val)],\n",
    "                          verbose=500,\n",
    "                          eval_metric='mae',\n",
    "                          early_stopping_rounds=50)\n",
    "\n",
    "    pred_val = lgb_model.predict(\n",
    "        X_val, num_iteration=lgb_model.best_iteration_)\n",
    "    df_oof = df_train.iloc[val_idx][['SaleID', ycol]].copy()\n",
    "    df_oof['pred'] = pred_val\n",
    "    oof.append(df_oof)\n",
    "\n",
    "    pred_test = lgb_model.predict(\n",
    "        df_test[feature_names], num_iteration=lgb_model.best_iteration_)\n",
    "    prediction['price'] += pred_test / 5\n",
    "\n",
    "    df_importance = pd.DataFrame({\n",
    "        'column': feature_names,\n",
    "        'importance': lgb_model.feature_importances_,\n",
    "    })\n",
    "    df_importance_list.append(df_importance)\n",
    "\n",
    "    del lgb_model, pred_val, pred_test, X_train, Y_train, X_val, Y_val\n",
    "    gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-17T08:08:36.127113Z",
     "start_time": "2020-03-17T08:08:36.116339Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>column</th>\n",
       "      <th>importance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>regionCode_price_mean</td>\n",
       "      <td>28482.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>regionCode</td>\n",
       "      <td>27488.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>name_price_mean</td>\n",
       "      <td>20725.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>v_3</td>\n",
       "      <td>15975.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>v_14</td>\n",
       "      <td>15801.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>car_age_day</td>\n",
       "      <td>15298.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>v_1</td>\n",
       "      <td>15170.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>v_11</td>\n",
       "      <td>15132.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>v_0</td>\n",
       "      <td>14905.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>v_8</td>\n",
       "      <td>14896.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>name</td>\n",
       "      <td>14884.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>v_7</td>\n",
       "      <td>14585.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>v_6</td>\n",
       "      <td>13227.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>v_2</td>\n",
       "      <td>13101.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>v_12</td>\n",
       "      <td>12838.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>power</td>\n",
       "      <td>12239.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>v_10</td>\n",
       "      <td>12100.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>v_9</td>\n",
       "      <td>11675.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>v_4</td>\n",
       "      <td>11556.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>model_price_mean</td>\n",
       "      <td>10859.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>v_5</td>\n",
       "      <td>10210.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>v_13</td>\n",
       "      <td>10138.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>name_count</td>\n",
       "      <td>10009.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>car_age_year</td>\n",
       "      <td>7905.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>model</td>\n",
       "      <td>6496.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>kilometer</td>\n",
       "      <td>5372.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>brand</td>\n",
       "      <td>5077.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>regDate_year</td>\n",
       "      <td>3832.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>bodyType</td>\n",
       "      <td>2081.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>notRepairedDamage</td>\n",
       "      <td>2018.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>fuelType</td>\n",
       "      <td>1588.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>gearbox</td>\n",
       "      <td>1228.6</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   column  importance\n",
       "0   regionCode_price_mean     28482.2\n",
       "1              regionCode     27488.0\n",
       "2         name_price_mean     20725.4\n",
       "3                     v_3     15975.8\n",
       "4                    v_14     15801.8\n",
       "5             car_age_day     15298.2\n",
       "6                     v_1     15170.2\n",
       "7                    v_11     15132.0\n",
       "8                     v_0     14905.8\n",
       "9                     v_8     14896.8\n",
       "10                   name     14884.2\n",
       "11                    v_7     14585.8\n",
       "12                    v_6     13227.2\n",
       "13                    v_2     13101.2\n",
       "14                   v_12     12838.0\n",
       "15                  power     12239.8\n",
       "16                   v_10     12100.2\n",
       "17                    v_9     11675.8\n",
       "18                    v_4     11556.6\n",
       "19       model_price_mean     10859.2\n",
       "20                    v_5     10210.8\n",
       "21                   v_13     10138.2\n",
       "22             name_count     10009.8\n",
       "23           car_age_year      7905.6\n",
       "24                  model      6496.0\n",
       "25              kilometer      5372.6\n",
       "26                  brand      5077.6\n",
       "27           regDate_year      3832.2\n",
       "28               bodyType      2081.6\n",
       "29      notRepairedDamage      2018.2\n",
       "30               fuelType      1588.4\n",
       "31                gearbox      1228.6"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_importance = pd.concat(df_importance_list)\n",
    "df_importance = df_importance.groupby(['column'])['importance'].agg(\n",
    "    'mean').sort_values(ascending=False).reset_index()\n",
    "df_importance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-17T08:08:36.139918Z",
     "start_time": "2020-03-17T08:08:36.128126Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mae: 483.59994168724864\n"
     ]
    }
   ],
   "source": [
    "df_oof = pd.concat(oof)\n",
    "df_oof[ycol] = np.expm1(df_oof[ycol])\n",
    "df_oof['pred'] = np.expm1(df_oof['pred'])\n",
    "mae = mean_absolute_error(df_oof[ycol], df_oof['pred'])\n",
    "print('mae:', mae)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-17T08:08:36.321477Z",
     "start_time": "2020-03-17T08:08:36.141839Z"
    }
   },
   "outputs": [],
   "source": [
    "prediction['price'] = np.expm1(prediction['price'])\n",
    "sub = prediction.copy(deep=True)\n",
    "sub.to_csv('sub/{}.csv'.format(mae), index=False, encoding='utf-8')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:dm] *",
   "language": "python",
   "name": "conda-env-dm-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
